commit be6ba7d2aa8d20944132ee54031b553c34d385db
Author: Eduard Zingerman <[email protected]>

bpf: correct stack liveness for tail calls

This patch updates bpf_insn_successors() to handle tail calls by creating
a jump table that reflects control flow can skip instructions between the
tail call and function exit when the tail call succeeds.

> diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
> index 5441341f1..8d0b60fa5 100644
> --- a/include/linux/bpf_verifier.h
> +++ b/include/linux/bpf_verifier.h

[ ... ]

> @@ -652,6 +652,7 @@ struct bpf_subprog_info {
>  	u32 start; /* insn idx of function entry point */
>  	u32 linfo_idx; /* The idx to the main_prog->aux->linfo */
>  	u32 postorder_start; /* The idx to the env->cfg.insn_postorder */
> +	u32 exit_idx; /* Index of one of the BPF_EXIT instructions in this subprogram */
>  	u16 stack_depth; /* max. stack depth used by this function */

[ ... ]

> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index 117a2b1cf..f564150ec 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -3555,8 +3555,12 @@ static int check_subprogs(struct bpf_verifier_env *env)
>  			subprog[cur_subprog].has_ld_abs = true;
>  		if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32)
>  			goto next;
> -		if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL)
> +		if (BPF_OP(code) == BPF_CALL)
>  			goto next;
> +		if (BPF_OP(code) == BPF_EXIT) {
> +			subprog[cur_subprog].exit_idx = i;
> +			goto next;
> +		}

[ ... ]

> +static int visit_tailcall_insn(struct bpf_verifier_env *env, int t)
> +{
> +	static struct bpf_subprog_info *subprog;
> +	struct bpf_iarray *jt;
> +
> +	if (env->insn_aux_data[t].jt)
> +		return 0;
> +
> +	jt = iarray_realloc(NULL, 2);
> +	if (!jt)
> +		return -ENOMEM;
> +
> +	subprog = bpf_find_containing_subprog(env, t);
> +	jt->items[0] = t + 1;
> +	jt->items[1] = subprog->exit_idx;
                        ^^^^^^^^^^^^^^^^^^

Can a subprogram with a tail_call have no BPF_EXIT instruction? The
verifier allows subprograms to end with an unconditional jump instead
of BPF_EXIT (see check_subprogs() which accepts BPF_JA as the last
instruction). In check_subprogs(), exit_idx is only set when a BPF_EXIT
is encountered. For subprograms ending with only jumps (like infinite
loops with tail_calls), exit_idx remains 0 from the kvzalloc() in
bpf_check().

If visit_tailcall_insn() uses exit_idx when it's 0, the jump table will
incorrectly point to instruction 0 instead of the actual exit, potentially
causing incorrect liveness analysis for stack slots.

> +	env->insn_aux_data[t].jt = jt;
> +	return 0;
> +}